import requests
from datetime import datetime
from lxml import etree
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess
SPIDERNAME='中国棉花协会'
def parseChinaCottonPubTime(time_string):
    #s='2020-06-05'
    time_string=time_string.strip()+' 00:00:00'
    struct_time = datetime.strptime(time_string, '%Y-%m-%d %H:%M:%S')
    return struct_time.strftime('%Y-%m-%d %H:%M:%S')

def parseChinaCottonContent(url):
    r=sess.get(url)
    r.encoding='utf8'
    selector=etree.HTML(r.text)
    ele=selector.cssselect(".txt")[0]
    content=ele.xpath("string(.)").strip().strip('\n')
    # print(content)
    return content


def getChinaCottonArticleList(articleCol,BeCrawledUrlList):
    url_1='http://www.china-cotton.org/advise/china_cotton_list'  #国内棉花资讯
    url_2='http://www.china-cotton.org/warning/newContent'  #棉花预警
    
    temp_article_ls=[]
    ####一下代码未修改未完成
    for url in (url_1,url_2):
        r=sess.get(url)
        r.encoding='utf8'
        selector=etree.HTML(r.text)
        eleList=selector.cssselect(".special div")
        for ele in  eleList:
            articleUrl=ele.xpath("./a/@href")[0]
            if 'javascript' in articleUrl:continue  #付费内容
            articleUrl='http://www.china-cotton.org'+articleUrl[2:]
            #判断是否已经爬取过，如果是，跳出循环
            if len(articleCol.search(where('url')==articleUrl)):break
            title=ele.xpath("./a/text()")[0].strip()
            publicTime=parseChinaCottonPubTime(ele.xpath("./span/text()")[0])
            temp_dict={'tags':['ChinaCotton'],'score':0,'uid':UID()}
            temp_dict['title']=title.strip()
            temp_dict['articleFrom']='中国棉花协会'
            temp_dict['url']=articleUrl.strip()
            temp_dict['publicTime']=publicTime.strip()
             #文章内容
            content=parseChinaCottonContent(articleUrl)
            temp_dict['content']=content
            #定文章所属期货品种,板块   棉花协会，都是棉花
            # n=parseContentToName(title+content)
            
            temp_dict['product_name']='棉花'
            temp_dict['group']='农产品'
          
            temp_article_ls.append(temp_dict)
    
     #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)
    